package com.gxljc.bear.crawler.itaogao.guochengxin;

import java.io.Serializable;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;

import org.apache.commons.collections.CollectionUtils;
import org.apache.log4j.Logger;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

import com.gxljc.bear.crawler.base.DolphinCrawlerConsts;
import com.gxljc.bear.crawler.base.DolphinFetchData;
import com.gxljc.bear.crawler.itaogao.ItaogaoConst;
import com.gxljc.bear.crawler.itaogao.NewsPageTable;
import com.gxljc.bear.crawler.itaogao.NewsPageTemplate;
import com.gxljc.bear.crawler.itaogao.util.MongodbUtil;
import com.gxljc.bear.crawler.util.HtmlUtil;
import com.gxljc.bear.crawler.util.MD5Util;

/**
 * 国承信Seed爬虫
 * 
 * @author tanghaitao
 * 
 * @since 2021年11月23日
 */
public class GuochengxinSeedCrawler implements Serializable {

	private static final long serialVersionUID = 1L;

	public static Logger LOG = Logger.getLogger(GuochengxinSeedCrawler.class);
	private Boolean proxy = false;

	public GuochengxinSeedCrawler(Boolean proxy) {
		this.proxy = proxy;
	}

	public void crawl() throws Exception {
		List<String> seeds = getSeed();
		if (CollectionUtils.isEmpty(seeds)) {
			LOG.error("page is empty");
			return;
		}
		crawlNoSpark(seeds);
	}

	public void crawlNoSpark(List<String> seeds) {
		for (String url : seeds) {
			try {
				crawl(url);
			} catch (Exception e) {
				LOG.error(String.format("crawlNoSpark error,seeds=%s", seeds), e);
			}
		}
	}

	// 单条url入口
	public int crawl(String url) throws Exception {
		LOG.info("crawl url = " + url);
		DolphinFetchData fetchData = DolphinFetchData.getInstance(DolphinCrawlerConsts.CrawlerChannelType.bear
				.getName());
		byte[] htmlByte = fetchData.getHtml(url, proxy, ItaogaoConst.CRAWL_TIMEOUT);
		if (htmlByte == null) {
			LOG.error("page result is empty url = " + url);
			return DolphinCrawlerConsts.RESULT_NO;
		}
		String html = new String(htmlByte, "utf-8");
		List<NewsPageTable> tables = extract(html, url);
		MongodbUtil.saveMongodb(tables, ItaogaoConst.MONGODB_TABLE.GUOCHENGXIN.getValue());
		return DolphinCrawlerConsts.RESULT_YES;
	}

	private List<NewsPageTable> extract(String result, String url) throws Exception {
		NewsPageTemplate template = new NewsPageTemplate();
		List<String> crawlUrlList = extractCrawlUrl(result);
		template.setCrawlUrl(crawlUrlList);
		return parseObject(template, url);
	}

	private List<String> extractCrawlUrl(String result) {
		List<String> crawlUrlList = new ArrayList<>();
		Document doc = Jsoup.parse(result);
		Elements list = doc.select(".item h2 a");
		for (Element l : list)
			crawlUrlList.add(l.select("a[href]").attr("href"));
		return crawlUrlList;
	}

	private List<NewsPageTable> parseObject(NewsPageTemplate template, String url) throws Exception {
		List<NewsPageTable> tables = new LinkedList<>();
		List<String> crawlUrls = template.getCrawlUrl();
		if (CollectionUtils.isEmpty(crawlUrls))
			return null;
		int size = crawlUrls.size();
		for (int i = 0; i < size; i++) {
			String crawlUrl = crawlUrls.get(i);
			NewsPageTable table = new NewsPageTable();
			crawlUrl = HtmlUtil.pictureUrlPref(crawlUrl, url);
			table.setId(genId(crawlUrl));
			table.setCrawlFlag(0);
			table.setCrawlUrl(crawlUrl);
			tables.add(table);
		}
		return tables;
	}

	// 生成id
	private String genId(String href) {
		return MD5Util.getMd5(href);
	}

	// 构造URL seed
	@SuppressWarnings("rawtypes")
	public List<String> getSeed() throws Exception {
		Map<String, Integer> seeds = new HashMap<String, Integer>();
		seeds.put("http://www.guochengxin.com/today_list/index_%s.shtml", 1106);
		List<String> urls = new LinkedList<String>();
		for (Map.Entry seed : seeds.entrySet()) {
			int max = (Integer) seed.getValue();
			String urlFormat = (String) seed.getKey();
			for (int i = 1; i <= max; i++) {
				String url = String.format(urlFormat, i + "");
				urls.add(url);
			}
		}
		return urls;
	}

}